iptables -A INPUT -p tcp -m multiport --dports 6443,2379,2380,10250 -j ACCEPT

[root@master ~]#swapoff -a
[root@master ~]#
[root@master ~]# free -m
              total        used        free      shared  buff/cache   available
Mem:            976         366         135           6         474         393
Swap:             0           0           0

# swap 一栏为0，表示已经关闭了swap

# 设置加载br_netfilter模块
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter

# 开启bridge-nf-call-iptables ，设置所需的 sysctl 参数，参数在重新启动后保持不变
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables  = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward                 = 1
EOF

# 应用 sysctl 参数而不重新启动
sudo sysctl --system

# for ubuntu
apt install -y containerd.io

containerd config default > /etc/containerd/config.toml

[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
  ...
  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
    SystemdCgroup = true

systemctl restart containerd

# for ubuntu
apt install -y docker.io

# 修改docker进程管理器
vi /etc/docker/daemon.json
{
"exec-opts": ["native.cgroupdriver=systemd"]
}
systemctl daemon-reload && systemctl restart docker
docker info | grep -i cgroup

# 以Ubuntu系统为例

# 安装仓库依赖
sudo apt-get update
sudo apt-get install -y apt-transport-https ca-certificates curl

# use google registry
sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list

# or use aliyun registry
curl -s https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | sudo apt-key add -
tee /etc/apt/sources.list.d/kubernetes.list <<EOF 
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF

# 安装指定版本的kubeadm, kubelet, kubectl
apt-get update
apt-get install -y kubelet=1.24.2-00 kubeadm=1.24.2-00 kubectl=1.24.2-00

# 查询有哪些版本
apt-cache madison kubeadm

#!/bin/bash
Version=${Version:-1.24.2}
wget https://dl.k8s.io/release/v${Version}/bin/linux/amd64/kubeadm
wget https://dl.k8s.io/release/v${Version}/bin/linux/amd64/kubelet
wget https://dl.k8s.io/release/v${Version}/bin/linux/amd64/kubectl
chmod +x kubeadm kubelet kubectl
cp kubeadm kubelet kubectl /usr/bin/

# add kubelet serivce
cat > /lib/systemd/system/kubelet.service << EOF
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/home/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=/usr/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

mkdir -p /etc/systemd/system/kubelet.service.d
cat > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf << \EOF
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/default/kubelet
ExecStart=
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS
EOF


systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet

apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration

apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration

apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration

apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration

apiVersion: kubeadm.k8s.io/v1beta3
kind: JoinConfiguration

kubeadm config print init-defaults
kubeadm config print join-defaults

apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
bootstrapTokens:
  ...
nodeRegistration:
  ...

apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
networking:
  ...
etcd:
  ...
apiServer:
  extraArgs:
    ...
  extraVolumes:
    ...
...

kubeadm config print init-defaults > kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 1.2.3.4 # 修改为apiserver的IP 或者去掉localAPIEndpoint则会读取默认IP。
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  imagePullPolicy: IfNotPresent
  name: node
  taints: null
---
apiServer:
  certSANs:
  - lb.k8s.domain  # 添加额外的apiserver的域名
  - <vip/lb_ip>
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}   # 默认为coredns
etcd:
  local:
    dataDir: /data/etcd   # 修改etcd的存储盘目录
imageRepository: k8s.gcr.io  # 修改镜像仓库地址
controlPlaneEndpoint: lb.k8s.domain  # 修改控制面域名
kind: ClusterConfiguration
kubernetesVersion: 1.24.0  # k8s 版本
networking:
  dnsDomain: cluster.local
  serviceSubnet: 10.96.0.0/12
  podSubnet: 10.244.0.0/16  # 设置pod的IP范围
scheduler: {}
---
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd   # 设置为systemd

kubectl get cm -n kube-system kubeadm-config -oyaml

kubeadm config images pull

sudo kubeadm init --config kubeadm-config.yaml --upload-certs  --node-name <nodename>

...
You can now join any number of control-plane node by running the following command on each as a root:
    kubeadm join 192.168.0.200:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866 --control-plane --certificate-key f8902e114ef118304e561c3ecd4d0b543adc226b7a07f675f56564185ffe0c07

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use kubeadm init phase upload-certs to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:
    kubeadm join 192.168.0.200:6443 --token 9vr73a.a8uxyaju799qwdjv --discovery-token-ca-cert-hash sha256:7c2e69131a36ae2a042a339b33381c6d0d43887e2de83720eff5359e26aec866

kubeadm join <control-plane-endpoint>:6443 --token <token> \
--discovery-token-ca-cert-hash sha256:<hash> \
--control-plane --certificate-key <certificate-key> \
--node-name <nodename>

kubeadm join <control-plane-endpoint>:6443 --token <token> \
--discovery-token-ca-cert-hash sha256:<hash> \
--cri-socket /run/containerd/containerd.sock \
--node-name <nodename>

## 如果安装之后node的状态都改为ready，即为成功
wget https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
kubectl apply -f ./kube-flannel.yml
kubectl get nodes

  Warning  FailedCreatePodSandBox  4m6s                kubelet            Failed to create pod sandbox: rpc error: code = Unknown desc = failed to setup network for sandbox "300d9b570cc1e23b6335c407b8e7d0ef2c74dc2fe5d7a110678c2dc919c62edf": plugin type="flannel" failed (add): failed to delegate add: failed to set bridge addr: "cni0" already has an IP address different from 10.244.3.1/24

# ifconfig cni0 |grep -w inet
        inet 10.244.5.1  netmask 255.255.255.0  broadcast 10.244.116.255

# cat /run/flannel/subnet.env
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.116.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true

ifconfig cni0 down    
ip link delete cni0

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

kubectl -n kubernetes-dashboard create token admin-user

# kubeadm重置
kubeadm reset

# 清空数据目录
rm -fr /data/etcd
rm -fr /etc/kubernetes
rm -fr ~/.kube/

ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
rm -f /etc/cni/net.d/*

[discovery] The cluster-info ConfigMap does not yet contain a JWS signature for token ID "abcdef", will try again

# 重新生成token
kubeadm token create --print-join-command
kubeadm token list

# kubeadm token create
oumnnc.aqlxuvdbntlvzoiv

# 重新生成hash
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

# 查看cluster-info
kubectl -n kube-public get configmaps cluster-info -o yaml

# 修改cluster-info
kubectl -n kube-public edit configmaps cluster-info

clusters:
- cluster:
    certificate-authority-data: xxx
    server: https://lb.k8s.domain:36443
  name: ""

[preflight] Some fatal errors occurred:
    [ERROR FileExisting-conntrack]: conntrack not found in system path

apt -y install conntrack

Error: failed to run Kubelet: unable to determine runtime API version: rpc error:code = Unavailable desc = connection error: desc = "transport: Error while dialing dial unix: missing address" 

# 查看启动参数是否遗漏，比如10-kubeadm.conf 文件参数缺失
systemctl cat --no-pager kubelet

cat /lib/systemd/system/kubelet.service

cat /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

ansible>=2.4.0
jinja2>=2.9.6
netaddr
pbr>=1.6
ansible-modules-hashivault>=3.9.4
hvac

# 安装pip
yum -y install epel-release
yum -y install python-pip
# 安装python-netaddr
pip install netaddr

# Jinja 2.9 (or newer)
pip install --upgrade jinja2

systemctl stop firewalld
systemctl disable firewalld
iptables -F

- name: Stop if swap enabled
  assert:
    that: ansible_swaptotal_mb == 0
  when: kubelet_fail_swap_on|default(true)
  ignore_errors: "{{ ignore_assert_errors }}"

[root@master ~]#swapoff -a
[root@master ~]#
[root@master ~]# free -m
              total        used        free      shared  buff/cache   available
Mem:            976         366         135           6         474         393
Swap:             0           0           0

# swap 一栏为0，表示已经关闭了swap

- name: Stop if memory is too small for masters
  assert:
    that: ansible_memtotal_mb >= 1500
  ignore_errors: "{{ ignore_assert_errors }}"
  when: inventory_hostname in groups['kube-master']

- name: Stop if memory is too small for nodes
  assert:
    that: ansible_memtotal_mb >= 1024
  ignore_errors: "{{ ignore_assert_errors }}"
  when: inventory_hostname in groups['kube-node']

git clone https://github.com/kubernetes-incubator/kubespray.git

cd kubespray
# 复制一份配置进行修改
cp -rfp inventory/sample inventory/k8s
vi inventory/k8s/hosts.ini

# Configure 'ip' variable to bind kubernetes services on a
# different ip than the default iface
# 主机名             ssh登陆IP                        ssh用户名               ssh登陆密码                 机器IP          子网掩码
kube-master-0     ansible_ssh_host=172.16.94.140   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.140   mask=/24
kube-node-41      ansible_ssh_host=172.16.94.141   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.141   mask=/24
kube-node-42      ansible_ssh_host=172.16.94.142   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.142   mask=/24

# configure a bastion host if your nodes are not directly reachable
# bastion ansible_ssh_host=x.x.x.x

[kube-master]
kube-master-0

[etcd]
kube-master-0

[kube-node]
kube-node-41
kube-node-42

[k8s-cluster:children]
kube-node
kube-master

[calico-rr]

# 进入主目录
cd kubespray
# 执行部署命令
ansible-playbook -i inventory/k8s/hosts.ini cluster.yml -b -vvv

ansible-playbook -i inventory/k8s/hosts.ini reset.yml -b -vvv

PLAY RECAP *****************************************************************************
kube-master-0              : ok=309  changed=30   unreachable=0    failed=0
kube-node-41               : ok=203  changed=8    unreachable=0    failed=0
kube-node-42               : ok=203  changed=8    unreachable=0    failed=0
localhost                  : ok=2    changed=0    unreachable=0    failed=0

kubernetes/preinstall : Update package management cache (YUM) --------------------23.96s
/root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/main.yml:121 
kubernetes/master : Master | wait for the apiserver to be running ----------------23.44s
/root/gopath/src/kubespray/roles/kubernetes/master/handlers/main.yml:79 
kubernetes/preinstall : Install packages requirements ----------------------------20.20s
/root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/main.yml:203 
kubernetes/secrets : Check certs | check if a cert already exists on node --------13.94s
/root/gopath/src/kubespray/roles/kubernetes/secrets/tasks/check-certs.yml:17 
gather facts from all instances --------------------------------------------------9.98s
/root/gopath/src/kubespray/cluster.yml:25 
kubernetes/node : install | Compare host kubelet with hyperkube container --------9.66s
/root/gopath/src/kubespray/roles/kubernetes/node/tasks/install_host.yml:2 
kubernetes-apps/ansible : Kubernetes Apps | Start Resources -----------------------9.27s
/root/gopath/src/kubespray/roles/kubernetes-apps/ansible/tasks/main.yml:37 
kubernetes-apps/ansible : Kubernetes Apps | Lay Down KubeDNS Template ------------8.47s
/root/gopath/src/kubespray/roles/kubernetes-apps/ansible/tasks/kubedns.yml:3
download : Sync container ---------------------------------------------------------8.23s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:15 
kubernetes-apps/network_plugin/calico : Start Calico resources --------------------7.82s
/root/gopath/src/kubespray/roles/kubernetes-apps/network_plugin/calico/tasks/main.yml:2 
download : Download items ---------------------------------------------------------7.67s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Download items ---------------------------------------------------------7.48s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Sync container ---------------------------------------------------------7.35s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:15 
download : Download items ---------------------------------------------------------7.16s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
network_plugin/calico : Calico | Copy cni plugins from calico/cni container -------7.10s
/root/gopath/src/kubespray/roles/network_plugin/calico/tasks/main.yml:62 
download : Download items ---------------------------------------------------------7.04s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6
download : Download items ---------------------------------------------------------7.01s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Sync container ---------------------------------------------------------7.00s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:15 
download : Download items ---------------------------------------------------------6.98s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 
download : Download items ---------------------------------------------------------6.79s
/root/gopath/src/kubespray/roles/download/tasks/main.yml:6 

# kubectl get all --namespace=kube-system
NAME             DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
ds/calico-node   3         3         3         3            3           <none>          2h

NAME                          DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deploy/kube-dns               2         2         2            2           2h
deploy/kubedns-autoscaler     1         1         1            1           2h
deploy/kubernetes-dashboard   1         1         1            1           2h

NAME                                 DESIRED   CURRENT   READY     AGE
rs/kube-dns-79d99cdcd5               2         2         2         2h
rs/kubedns-autoscaler-5564b5585f     1         1         1         2h
rs/kubernetes-dashboard-69cb58d748   1         1         1         2h

NAME             DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
ds/calico-node   3         3         3         3            3           <none>          2h

NAME                          DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deploy/kube-dns               2         2         2            2           2h
deploy/kubedns-autoscaler     1         1         1            1           2h
deploy/kubernetes-dashboard   1         1         1            1           2h

NAME                                 DESIRED   CURRENT   READY     AGE
rs/kube-dns-79d99cdcd5               2         2         2         2h
rs/kubedns-autoscaler-5564b5585f     1         1         1         2h
rs/kubernetes-dashboard-69cb58d748   1         1         1         2h

NAME                                       READY     STATUS    RESTARTS   AGE
po/calico-node-22vsg                       1/1       Running   0          2h
po/calico-node-t7zgw                       1/1       Running   0          2h
po/calico-node-zqnx8                       1/1       Running   0          2h
po/kube-apiserver-kube-master-0            1/1       Running   0          22h
po/kube-controller-manager-kube-master-0   1/1       Running   0          2h
po/kube-dns-79d99cdcd5-f2t6t               3/3       Running   0          2h
po/kube-dns-79d99cdcd5-gw944               3/3       Running   0          2h
po/kube-proxy-kube-master-0                1/1       Running   2          22h
po/kube-proxy-kube-node-41                 1/1       Running   3          22h
po/kube-proxy-kube-node-42                 1/1       Running   3          22h
po/kube-scheduler-kube-master-0            1/1       Running   0          2h
po/kubedns-autoscaler-5564b5585f-lt9bb     1/1       Running   0          2h
po/kubernetes-dashboard-69cb58d748-wmb9x   1/1       Running   0          2h
po/nginx-proxy-kube-node-41                1/1       Running   3          22h
po/nginx-proxy-kube-node-42                1/1       Running   3          22h

NAME                       TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)         AGE
svc/kube-dns               ClusterIP   10.233.0.3     <none>        53/UDP,53/TCP   2h
svc/kubernetes-dashboard   ClusterIP   10.233.27.24   <none>        443/TCP         2h

# kubectl get nodes
NAME            STATUS    ROLES     AGE       VERSION
kube-master-0   Ready     master    22h       v1.9.5
kube-node-41    Ready     node      22h       v1.9.5
kube-node-42    Ready     node      22h       v1.9.5

# kubectl get cs
NAME                 STATUS    MESSAGE              ERROR
scheduler            Healthy   ok
controller-manager   Healthy   ok
etcd-0               Healthy   {"health": "true"}

# Configure 'ip' variable to bind kubernetes services on a
# different ip than the default iface
# 主机名             ssh登陆IP                        ssh用户名               ssh登陆密码                 机器IP          子网掩码
kube-master-0     ansible_ssh_host=172.16.94.140   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.140   mask=/24
kube-node-41      ansible_ssh_host=172.16.94.141   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.141   mask=/24
kube-node-42      ansible_ssh_host=172.16.94.142   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.142   mask=/24
kube-node-43      ansible_ssh_host=172.16.94.143   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.143   mask=/24

# configure a bastion host if your nodes are not directly reachable
# bastion ansible_ssh_host=x.x.x.x

[kube-master]
kube-master-0

[etcd]
kube-master-0

[kube-node]
kube-node-41
kube-node-42
kube-node-43

[k8s-cluster:children]
kube-node
kube-master

[calico-rr]

# 进入主目录
cd kubespray
# 执行部署命令
ansible-playbook -i inventory/k8s/hosts.ini scale.yml -b -vvv

PLAY RECAP ***************************************
kube-node-41               : ok=228  changed=11   unreachable=0    failed=0
kube-node-42               : ok=197  changed=6    unreachable=0    failed=0
kube-node-43               : ok=227  changed=69   unreachable=0    failed=0 # 新增Node节点
localhost                  : ok=2    changed=0    unreachable=0    failed=0

# kubectl get nodes
NAME            STATUS    ROLES     AGE       VERSION
kube-master-0   Ready     master    1d        v1.9.5
kube-node-41    Ready     node      1d        v1.9.5
kube-node-42    Ready     node      1d        v1.9.5
kube-node-43    Ready     node      1m        v1.9.5   #该节点为新增Node节点

# kubectl get po --namespace=kube-system -o wide
NAME                                    READY     STATUS    RESTARTS   AGE       IP               NODE
calico-node-22vsg                       1/1       Running   0          10h       172.16.94.140    kube-master-0
calico-node-8fz9x                       1/1       Running   2          27m       172.16.94.143    kube-node-43
calico-node-t7zgw                       1/1       Running   0          10h       172.16.94.142    kube-node-42
calico-node-zqnx8                       1/1       Running   0          10h       172.16.94.141    kube-node-41
kube-apiserver-kube-master-0            1/1       Running   0          1d        172.16.94.140    kube-master-0
kube-controller-manager-kube-master-0   1/1       Running   0          10h       172.16.94.140    kube-master-0
kube-dns-79d99cdcd5-f2t6t               3/3       Running   0          10h       10.233.100.194   kube-node-41
kube-dns-79d99cdcd5-gw944               3/3       Running   0          10h       10.233.107.1     kube-node-42
kube-proxy-kube-master-0                1/1       Running   2          1d        172.16.94.140    kube-master-0
kube-proxy-kube-node-41                 1/1       Running   3          1d        172.16.94.141    kube-node-41
kube-proxy-kube-node-42                 1/1       Running   3          1d        172.16.94.142    kube-node-42
kube-proxy-kube-node-43                 1/1       Running   0          26m       172.16.94.143    kube-node-43
kube-scheduler-kube-master-0            1/1       Running   0          10h       172.16.94.140    kube-master-0
kubedns-autoscaler-5564b5585f-lt9bb     1/1       Running   0          10h       10.233.100.193   kube-node-41
kubernetes-dashboard-69cb58d748-wmb9x   1/1       Running   0          10h       10.233.107.2     kube-node-42
nginx-proxy-kube-node-41                1/1       Running   3          1d        172.16.94.141    kube-node-41
nginx-proxy-kube-node-42                1/1       Running   3          1d        172.16.94.142    kube-node-42
nginx-proxy-kube-node-43                1/1       Running   0          26m       172.16.94.143    kube-node-43

ansible-playbook -i inventory/k8s/hosts.ini cluster.yml -b -vvv

# Configure 'ip' variable to bind kubernetes services on a
# different ip than the default iface
# 主机名             ssh登陆IP                        ssh用户名               ssh登陆密码                 机器IP          子网掩码
kube-master-0     ansible_ssh_host=172.16.94.140   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.140   mask=/24
kube-master-1     ansible_ssh_host=172.16.94.144   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.144   mask=/24
kube-master-2     ansible_ssh_host=172.16.94.145   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.145   mask=/24
kube-node-41      ansible_ssh_host=172.16.94.141   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.141   mask=/24
kube-node-42      ansible_ssh_host=172.16.94.142   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.142   mask=/24
kube-node-43      ansible_ssh_host=172.16.94.143   ansible_ssh_user=root   ansible_ssh_pass=123  ip=172.16.94.143   mask=/24

# configure a bastion host if your nodes are not directly reachable
# bastion ansible_ssh_host=x.x.x.x

[kube-master]
kube-master-0
kube-master-1
kube-master-2

[etcd]
kube-master-0
kube-master-1
kube-master-2

[kube-node]
kube-node-41
kube-node-42
kube-node-43

[k8s-cluster:children]
kube-node
kube-master

[calico-rr]

ansible-playbook upgrade-cluster.yml -b -i inventory/k8s/hosts.ini -e kube_version=v1.10.4 -vvv

fatal: [node1]: FAILED! => {"failed": true, "msg": "The ipaddr filter requires python-netaddr be installed on the ansible controller"}

fatal: [kube-master-0]: FAILED! => {
    "assertion": "ansible_swaptotal_mb == 0",
    "changed": false,
    "evaluated_to": false
}
fatal: [kube-node-41]: FAILED! => {
    "assertion": "ansible_swaptotal_mb == 0",
    "changed": false,
    "evaluated_to": false
}
fatal: [kube-node-42]: FAILED! => {
    "assertion": "ansible_swaptotal_mb == 0",
    "changed": false,
    "evaluated_to": false
}

TASK [kubernetes/preinstall : Stop if memory is too small for masters] *********************************************************************************************************************************************************************************************************
task path: /root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/verify-settings.yml:52
Friday 10 August 2018  21:50:26 +0800 (0:00:00.940)       0:01:14.088 *********
fatal: [kube-master-0]: FAILED! => {
    "assertion": "ansible_memtotal_mb >= 1500",
    "changed": false,
    "evaluated_to": false
}

TASK [kubernetes/preinstall : Stop if memory is too small for nodes] ***********************************************************************************************************************************************************************************************************
task path: /root/gopath/src/kubespray/roles/kubernetes/preinstall/tasks/verify-settings.yml:58
Friday 10 August 2018  21:50:27 +0800 (0:00:00.570)       0:01:14.659 *********
fatal: [kube-node-41]: FAILED! => {
    "assertion": "ansible_memtotal_mb >= 1024",
    "changed": false,
    "evaluated_to": false
}
fatal: [kube-node-42]: FAILED! => {
    "assertion": "ansible_memtotal_mb >= 1024",
    "changed": false,
    "evaluated_to": false
}
	to retry, use: --limit @/root/gopath/src/kubespray/cluster.retry

FAILED - RETRYING: Master | wait for kube-scheduler (1 retries left).
FAILED - RETRYING: Master | wait for kube-scheduler (1 retries left).
fatal: [node1]: FAILED! => {"attempts": 60, "changed": false, "content": "", "failed": true, "msg": "Status code was not [200]: Request failed: <urlopen error [Errno 111] Connection refused>", "redirected": false, "status": -1, "url": "http://localhost:10251/healthz"}

failed: [k8s-node-1] (item={u'name': u'docker-engine-1.13.1-1.el7.centos'}) => {
    "attempts": 4,
    "changed": false,
    ...
    "item": {
        "name": "docker-engine-1.13.1-1.el7.centos"
    },
    "msg": "Error: docker-ce-selinux conflicts with 2:container-selinux-2.66-1.el7.noarch\n",
    "rc": 1,
    "results": [
        "Loaded plugins: fastestmirror\nLoading mirror speeds from cached hostfile\n * elrepo: mirrors.tuna.tsinghua.edu.cn\n * epel: mirrors.tongji.edu.cn\nPackage docker-engine is obsoleted by docker-ce, trying to install docker-ce-17.03.2.ce-1.el7.centos.x86_64 instead\nResolving Dependencies\n--> Running transaction check\n---> Package docker-ce.x86_64 0:17.03.2.ce-1.el7.centos will be installed\n--> Processing Dependency: docker-ce-selinux >= 17.03.2.ce-1.el7.centos for package: docker-ce-17.03.2.ce-1.el7.centos.x86_64\n--> Processing Dependency: libltdl.so.7()(64bit) for package: docker-ce-17.03.2.ce-1.el7.centos.x86_64\n--> Running transaction check\n---> Package docker-ce-selinux.noarch 0:17.03.2.ce-1.el7.centos will be installed\n---> Package libtool-ltdl.x86_64 0:2.4.2-22.el7_3 will be installed\n--> Processing Conflict: docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch conflicts docker-selinux\n--> Restarting Dependency Resolution with new changes.\n--> Running transaction check\n---> Package container-selinux.noarch 2:2.55-1.el7 will be updated\n---> Package container-selinux.noarch 2:2.66-1.el7 will be an update\n--> Processing Conflict: docker-ce-selinux-17.03.2.ce-1.el7.centos.noarch conflicts docker-selinux\n--> Finished Dependency Resolution\n You could try using --skip-broken to work around the problem\n You could try running: rpm -Va --nofiles --nodigest\n"
    ]
}

sudo yum remove -y docker \
                  docker-client \
                  docker-client-latest \
                  docker-common \
                  docker-latest \
                  docker-latest-logrotate \
                  docker-logrotate \
                  docker-selinux \
                  docker-engine-selinux \
                  docker-engine

curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/

curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.9.0/bin/linux/amd64/kubectl && chmod +x kubectl && sudo mv kubectl /usr/local/bin/

curl -Lo minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/

curl -Lo minikube https://storage.googleapis.com/minikube/releases/v0.28.2/minikube-linux-amd64 && chmod +x minikube && sudo mv minikube /usr/local/bin/

Minikube is a CLI tool that provisions and manages single-node Kubernetes clusters optimized for development workflows.

Usage:
  minikube [command]

Available Commands:
  addons           Modify minikube's kubernetes addons
  cache            Add or delete an image from the local cache.
  completion       Outputs minikube shell completion for the given shell (bash or zsh)
  config           Modify minikube config
  dashboard        Opens/displays the kubernetes dashboard URL for your local cluster
  delete           Deletes a local kubernetes cluster
  docker-env       Sets up docker env variables; similar to '$(docker-machine env)'
  get-k8s-versions Gets the list of Kubernetes versions available for minikube when using the localkube bootstrapper
  ip               Retrieves the IP address of the running cluster
  logs             Gets the logs of the running localkube instance, used for debugging minikube, not user code
  mount            Mounts the specified directory into minikube
  profile          Profile sets the current minikube profile
  service          Gets the kubernetes URL(s) for the specified service in your local cluster
  ssh              Log into or run a command on a machine with SSH; similar to 'docker-machine ssh'
  ssh-key          Retrieve the ssh identity key path of the specified cluster
  start            Starts a local kubernetes cluster
  status           Gets the status of a local kubernetes cluster
  stop             Stops a running local kubernetes cluster
  update-check     Print current and latest version number
  update-context   Verify the IP address of the running cluster in kubeconfig.
  version          Print the version of minikube

Flags:
      --alsologtostderr                  log to standard error as well as files
  -b, --bootstrapper string              The name of the cluster bootstrapper that will set up the kubernetes cluster. (default "localkube")
      --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
      --log_dir string                   If non-empty, write log files in this directory
      --loglevel int                     Log level (0 = DEBUG, 5 = FATAL) (default 1)
      --logtostderr                      log to standard error instead of files
  -p, --profile string                   The name of the minikube VM being used.
	This can be modified to allow for multiple minikube instances to be run independently (default "minikube")
      --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
  -v, --v Level                          log level for V logs
      --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging

Use "minikube [command] --help" for more information about a command.

minikube start --vm-driver=none

root@ubuntu:~# minikube start --vm-driver=none
Starting local Kubernetes v1.10.0 cluster...
Starting VM...
Getting VM IP address...
Moving files into cluster...
Downloading kubeadm v1.10.0
Downloading kubelet v1.10.0
^[[DFinished Downloading kubelet v1.10.0
Finished Downloading kubeadm v1.10.0
Setting up certs...
Connecting to cluster...
Setting up kubeconfig...
Starting cluster components...
Kubectl is now configured to use the cluster.
===================
WARNING: IT IS RECOMMENDED NOT TO RUN THE NONE DRIVER ON PERSONAL WORKSTATIONS
	The 'none' driver will run an insecure kubernetes apiserver as root that may leave the host vulnerable to CSRF attacks

When using the none driver, the kubectl config and credentials generated will be root owned and will appear in the root home directory.
You will need to move the files to the appropriate location and then set the correct permissions.  An example of this is below:

	sudo mv /root/.kube $HOME/.kube # this will write over any previous configuration
	sudo chown -R $USER $HOME/.kube
	sudo chgrp -R $USER $HOME/.kube

	sudo mv /root/.minikube $HOME/.minikube # this will write over any previous configuration
	sudo chown -R $USER $HOME/.minikube
	sudo chgrp -R $USER $HOME/.minikube

This can also be done automatically by setting the env var CHANGE_MINIKUBE_NONE_USER=true
Loading cached images from config file.

# 查阅版本
minikube get-k8s-versions
# 选择版本启动
minikube start --kubernetes-version v1.7.3 --vm-driver=none

$ minikube status
minikube: Running
cluster: Running
kubectl: Correctly Configured: pointing to minikube-vm at 172.16.94.139

root@ubuntu:~# kubectl get all --namespace=kube-system
NAME                                        READY     STATUS    RESTARTS   AGE
pod/etcd-minikube                           1/1       Running   0          38m
pod/kube-addon-manager-minikube             1/1       Running   0          38m
pod/kube-apiserver-minikube                 1/1       Running   1          39m
pod/kube-controller-manager-minikube        1/1       Running   0          38m
pod/kube-dns-86f4d74b45-bdfnx               3/3       Running   0          38m
pod/kube-proxy-dqdvg                        1/1       Running   0          38m
pod/kube-scheduler-minikube                 1/1       Running   0          38m
pod/kubernetes-dashboard-5498ccf677-c2gnh   1/1       Running   0          38m
pod/storage-provisioner                     1/1       Running   0          38m

NAME                           TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)         AGE
service/kube-dns               ClusterIP   10.96.0.10      <none>        53/UDP,53/TCP   38m
service/kubernetes-dashboard   NodePort    10.104.48.227   <none>        80:30000/TCP    38m

NAME                        DESIRED   CURRENT   READY     UP-TO-DATE   AVAILABLE   NODE SELECTOR   AGE
daemonset.apps/kube-proxy   1         1         1         1            1           <none>          38m

NAME                                   DESIRED   CURRENT   UP-TO-DATE   AVAILABLE   AGE
deployment.apps/kube-dns               1         1         1            1           38m
deployment.apps/kubernetes-dashboard   1         1         1            1           38m

NAME                                              DESIRED   CURRENT   READY     AGE
replicaset.apps/kube-dns-86f4d74b45               1         1         1         38m
replicaset.apps/kubernetes-dashboard-5498ccf677   1         1         1         38m

[root@minikube ~]# minikube start
Starting local Kubernetes v1.10.0 cluster...
Starting VM...
Downloading Minikube ISO
 160.27 MB / 160.27 MB [============================================] 100.00% 0s
E0727 15:47:08.655647    9407 start.go:174] Error starting host: Error creating host: Error executing step: Running precreate checks.
: VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path.

 Retrying.
E0727 15:47:08.656994    9407 start.go:180] Error starting host:  Error creating host: Error executing step: Running precreate checks.
: VBoxManage not found. Make sure VirtualBox is installed and VBoxManage is in the path
================================================================================
An error has occurred. Would you like to opt in to sending anonymized crash
information to minikube to help prevent future errors?
To opt out of these messages, run the command:
	minikube config set WantReportErrorPrompt false
================================================================================
Please enter your response [Y/n]:

[root@minikube ~]# minikube start --vm-driver=none
Starting local Kubernetes v1.10.0 cluster...
Starting VM...
E0727 15:56:54.936706    9441 start.go:174] Error starting host: Error creating host: Error executing step: Running precreate checks.
: docker cannot be found on the path for this machine. A docker installation is a requirement for using the none driver: exec: "docker": executable file not found in $PATH.

 Retrying.
E0727 15:56:54.938930    9441 start.go:180] Error starting host:  Error creating host: Error executing step: Running precreate checks.
: docker cannot be found on the path for this machine. A docker installation is a requirement for using the none driver: exec: "docker": executable file not found in $PATH

curl -Lo ./kind "https://github.com/kubernetes-sigs/kind/releases/download/v0.7.0/kind-$(uname)-amd64"
chmod +x ./kind
mv ./kind /some-dir-in-your-PATH/kind

$ kind create cluster
Creating cluster "kind" ...
 ✓ Ensuring node image (kindest/node:v1.17.0) 🖼
 ✓ Preparing nodes 📦
 ✓ Writing configuration 📜
 ✓ Starting control-plane 🕹️
 ✓ Installing CNI 🔌
 ✓ Installing StorageClass 💾
Set kubectl context to "kind-kind"
You can now use your cluster with:

kubectl cluster-info --context kind-kind

Not sure what to do next? 😅 Check out https://kind.sigs.k8s.io/docs/user/quick-start/

$ kubectl cluster-info --context kind-kind
Kubernetes master is running at https://127.0.0.1:32768
KubeDNS is running at https://127.0.0.1:32768/api/v1/namespaces/kube-system/services/kube-dns:dns/proxy

$ kubectl get node -o wide
NAME                 STATUS   ROLES    AGE   VERSION   INTERNAL-IP   EXTERNAL-IP   OS-IMAGE       KERNEL-VERSION                      CONTAINER-RUNTIME
kind-control-plane   Ready    master   35h   v1.17.0   172.17.0.2    <none>        Ubuntu 19.10   3.10.107-1-tlinux2_kvm_guest-0049   containerd://1.3.2

$ kubectl get po --all-namespaces -o wide
NAMESPACE            NAME                                         READY   STATUS    RESTARTS   AGE   IP           NODE                 NOMINATED NODE   READINESS GATES
kube-system          coredns-6955765f44-lqk9v                     1/1     Running   0          35h   10.244.0.4   kind-control-plane   <none>           <none>
kube-system          coredns-6955765f44-zpsmc                     1/1     Running   0          35h   10.244.0.3   kind-control-plane   <none>           <none>
kube-system          etcd-kind-control-plane                      1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kindnet-8mt7d                                1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-apiserver-kind-control-plane            1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-controller-manager-kind-control-plane   1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-proxy-5w25s                             1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
kube-system          kube-scheduler-kind-control-plane            1/1     Running   0          35h   172.17.0.2   kind-control-plane   <none>           <none>
local-path-storage   local-path-provisioner-7745554f7f-dckzr      1/1     Running   0          35h   10.244.0.2   kind-control-plane   <none>           <none>

$ docker ps
CONTAINER ID        IMAGE                  COMMAND                  CREATED             STATUS              PORTS                       NAMES
93b291f99dd4        kindest/node:v1.17.0   "/usr/local/bin/entr…"   2 minutes ago       Up 2 minutes        127.0.0.1:32768->6443/tcp   kind-control-plane

$ docker exec -it 93b291f99dd4 bash
root@kind-control-plane:/# ps auxw
USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
root         1  0.1  0.0  19512  7480 ?        Ss   03:18   0:00 /sbin/init
root       105  0.0  0.0  26396  7344 ?        S<s  03:18   0:00 /lib/systemd/systemd-journald
root       141  2.3  0.3 2374736 51564 ?       Ssl  03:18   0:06 /usr/local/bin/containerd
root       325  0.0  0.0 112540  5036 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 3f415d609e15ef12b9f53557891c311c156b912d5a326544a25c8b29cfa9d366 -address /run/containerd/containerd.sock
root       346  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       370  0.0  0.0 112540  5108 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 1e1f3eed09f701fb621325e7b9e96d1c3de60ebd3bd64e0aec376e9490cf0e57 -address /run/containerd/containerd.sock
root       397  0.0  0.0 112540  4684 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id c5e451089a1a5b3dfb2cc68ee27ac7d414285be55ecfdc5bd59180fbfbc7df2e -address /run/containerd/containerd.sock
root       424  0.0  0.0 112540  4924 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 81e35f29ac8c2dda344125a10e3791be7ccf788a88f1efbc3397fa319f02881f -address /run/containerd/containerd.sock
root       443  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       458  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       465  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       548  0.7  0.1 145500 27724 ?        Ssl  03:18   0:02 kube-scheduler --authentication-kubeconfig=/etc/kubernetes/scheduler.conf --authorization-kubeconfig=/etc/kubernetes/scheduler.conf --bind-address=127.0.0.1 --kubeconfig=/etc/kubernetes/scheduler.conf --leader-elect=true
root       589  1.0  0.3 159536 54384 ?        Ssl  03:18   0:02 kube-controller-manager --allocate-node-cidrs=true --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf --bind-address=127.0.0.1 --client-ca-file=/etc/kubernetes/pki/ca.cr
root       613  3.8  1.6 445780 273484 ?       Ssl  03:18   0:10 kube-apiserver --advertise-address=172.17.0.2 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/
root       660  1.4  0.2 10613604 37448 ?      Ssl  03:18   0:04 etcd --advertise-client-urls=https://172.17.0.2:2379 --cert-file=/etc/kubernetes/pki/etcd/server.crt --client-cert-auth=true --data-dir=/var/lib/etcd --initial-advertise-peer-urls=https://172.17.0.2:2380 --initial-cluster=kind-control-plane=https://172.
root       718  1.3  0.3 2084848 52772 ?       Ssl  03:18   0:03 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf --config=/var/lib/kubelet/config.yaml --container-runtime=remote --container-runtime-endpoint=/run/containerd/containerd.sock --fail
root       876  0.0  0.0 112540  5084 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id adfbea8fec5ac6986407291f5bfc5aecead176954e5dabbe1517b98dd77bf78b -address /run/containerd/containerd.sock
root       893  0.0  0.0 112540  4796 ?        Sl   03:18   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 53bdce023626b60ffaa0548b5888e457dc9c3bc45c7808a385dd0f63dcc90327 -address /run/containerd/containerd.sock
root       924  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root       931  0.0  0.0   1012     4 ?        Ss   03:18   0:00 /pause
root      1000  0.0  0.0 127616 11100 ?        Ssl  03:18   0:00 /bin/kindnetd
root      1017  0.0  0.1 141060 19420 ?        Ssl  03:18   0:00 /usr/local/bin/kube-proxy --config=/var/lib/kube-proxy/config.conf --hostname-override=kind-control-plane
root      1066  0.0  0.0      0     0 ?        Z    03:18   0:00 [iptables-nft-sa] <defunct>
root      1080  0.0  0.0      0     0 ?        Z    03:18   0:00 [iptables-nft-sa] <defunct>
root      1241  0.0  0.0 112540  5156 ?        Sl   03:19   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 5cbd7bbe186cf5847786c7a03aa4c6f82e6c805d0a189f0f3e8fb1750594260d -address /run/containerd/containerd.sock
root      1262  0.0  0.0   1012     4 ?        Ss   03:19   0:00 /pause
root      1303  0.1  0.0 134372 14088 ?        Ssl  03:19   0:00 local-path-provisioner --debug start --helper-image k8s.gcr.io/debian-base:v2.0.0 --config /etc/config/config.json
root      1411  0.0  0.0 112540  4876 ?        Sl   03:19   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 196b440345cb5ef47a6c31222323d35bbfef85d1d79c149ec0e3a6e22022a5f0 -address /run/containerd/containerd.sock
root      1437  0.0  0.0   1012     4 ?        Ss   03:19   0:00 /pause
root      1450  0.0  0.0 112540  4380 ?        Sl   03:19   0:00 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id de7bdf052083978c78708383f842567d4fb38adff22a56792437a4de82425afe -address /run/containerd/containerd.sock
root      1480  0.0  0.0   1012     4 ?        Ss   03:19   0:00 /pause
root      1530  0.1  0.1 144324 19056 ?        Ssl  03:19   0:00 /coredns -conf /etc/coredns/Corefile
root      1531  0.1  0.1 144580 19204 ?        Ssl  03:19   0:00 /coredns -conf /etc/coredns/Corefile

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: v1
kind: Secret
metadata:
  name: admin-user-secret
  namespace: kubernetes-dashboard
  annotations:
    kubernetes.io/service-account.name: "admin-user"   
type: kubernetes.io/service-account-token  

kubectl apply -f dashboard-adminuser.yaml 

kubectl -n kubernetes-dashboard create token admin-user --duration 8760h

kubectl get secret admin-user-secret -n kubernetes-dashboard -o jsonpath={".data.token"} | base64 -d

kubectl -n kubernetes-dashboard delete serviceaccount admin-user
kubectl -n kubernetes-dashboard delete clusterrolebinding admin-user

kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: <namespace>
  name: <namespace>-admin-role
rules:
  - apiGroups:
    - '*'
    resources:
    - '*'
    verbs:
    - '*'

apiVersion: v1
kind: ServiceAccount
metadata:
  name: <namespace>-admin-user
  namespace: <namespace>

apiVersion: v1
kind: Secret
metadata:
  name: ${SecretName}
  namespace: ${ServiceAccountNS}
  annotations:
    kubernetes.io/service-account.name: "${ServiceAccountName}"   
type: kubernetes.io/service-account-token

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: <namespace>-admin-user
  namespace: <namespace>
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: <namespace>-admin-role
subjects:
- kind: ServiceAccount
  name: <namespace>-admin-user
  namespace: <namespace>

kubectl -n <namespace> create token <ServiceAccount> --duration 8760h

kubectl get secret ${SecretName} -n ${ServiceAccountNS} -o jsonpath={".data.token"} | base64 -d

apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: <namespace>-admin-user
  namespace: <namespace>
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: view
subjects:
- kind: ServiceAccount
  name: <namespace>-admin-user
  namespace: <namespace>

kubeadm upgrade plan

# kubeadm的版本为v1.26.7
./kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"26", GitVersion:"v1.26.7", GitCommit:"84e1fc493a47446df2e155e70fca768d2653a398", GitTreeState:"clean", BuildDate:"2023-07-19T12:22:13Z", GoVersion:"go1.20.6", Compiler:"gc", Platform:"linux/amd64"}

# 当前k8s集群版本1.24.2 版本跨度过大。
./kubeadm upgrade plan
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[upgrade/config] FATAL: this version of kubeadm only supports deploying clusters with the control plane version >= 1.25.0. Current version: v1.24.2
To see the stack trace of this error execute with --v=5 or higher

./kubeadm upgrade plan
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade] Fetching available versions to upgrade to
[upgrade/versions] Cluster version: v1.24.2
[upgrade/versions] kubeadm version: v1.25.12
I0815 21:41:34.096199 2934255 version.go:256] remote version is much newer: v1.27.4; falling back to: stable-1.25
[upgrade/versions] Target version: v1.25.12
[upgrade/versions] Latest version in the v1.24 series: v1.24.16

Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT   CURRENT       TARGET
kubelet     4 x v1.24.2   v1.24.16

Upgrade to the latest version in the v1.24 series:

COMPONENT                 CURRENT   TARGET
kube-apiserver            v1.24.2   v1.24.16
kube-controller-manager   v1.24.2   v1.24.16
kube-scheduler            v1.24.2   v1.24.16
kube-proxy                v1.24.2   v1.24.16
CoreDNS                   v1.8.6    v1.9.3
etcd                      3.5.3-0   3.5.6-0

You can now apply the upgrade by executing the following command:

    kubeadm upgrade apply v1.24.16

_____________________________________________________________________

Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT   CURRENT       TARGET
kubelet     4 x v1.24.2   v1.25.12

Upgrade to the latest stable version:

COMPONENT                 CURRENT   TARGET
kube-apiserver            v1.24.2   v1.25.12
kube-controller-manager   v1.24.2   v1.25.12
kube-scheduler            v1.24.2   v1.25.12
kube-proxy                v1.24.2   v1.25.12
CoreDNS                   v1.8.6    v1.9.3
etcd                      3.5.3-0   3.5.6-0

You can now apply the upgrade by executing the following command:

    kubeadm upgrade apply v1.25.12

_____________________________________________________________________


The table below shows the current state of component configs as understood by this version of kubeadm.
Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or
resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually
upgrade to is denoted in the "PREFERRED VERSION" column.

API GROUP                 CURRENT VERSION   PREFERRED VERSION   MANUAL UPGRADE REQUIRED
kubeproxy.config.k8s.io   v1alpha1          v1alpha1            no
kubelet.config.k8s.io     v1beta1           v1beta1             no
_____________________________________________________________________

kubeadm upgrade apply v1.25.x -f

[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.25.x". Enjoy!

[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.

kubeadm upgrade node

# 将 <node-to-drain> 替换为你要腾空的控制面节点名称
kubectl drain <node-to-drain> --ignore-daemonsets

# 用最新的补丁版本替换 1.27.x-00 中的 x
apt-mark unhold kubelet kubectl && \
apt-get update && apt-get install -y kubelet=1.27.x-00 kubectl=1.27.x-00 && \
apt-mark hold kubelet kubectl

sudo systemctl daemon-reload
sudo systemctl restart kubelet

# 将 <node-to-uncordon> 替换为你的节点名称
kubectl uncordon <node-to-uncordon>

tmp/
├── kubeadm-backup-etcd-2023-08-16-14-50-50
│   └── etcd
└── kubeadm-backup-manifests-2023-08-16-14-50-50
    ├── etcd.yaml
    ├── kube-apiserver.yaml
    ├── kube-controller-manager.yaml
    └── kube-scheduler.yaml

"CreatePodSandbox for pod failed" err="open /run/systemd/resolve/resolv.conf: no such file or directory" pod="kube-system/kube-apiserver"

# 查看以下的resolv.conf是否存在
cat /var/lib/kubelet/config.yaml | grep resolvConf
/run/systemd/resolve/resolv.conf

# 如果不存在，检查systemd-resolved是否正常运行，
systemctl status systemd-resolved

# 如果没有运行，则运行该服务
systemctl start systemd-resolved

# 或者新建文件/run/systemd/resolve/resolv.conf，并将其他master的文件拷贝过来。

# 下载cfssl
$ curl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /usr/local/bin/cfssl
$ curl https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /usr/local/bin/cfssljson
$ curl https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -o /usr/local/bin/cfssl-certinfo

# 添加可执行权限
$ chmod +x /usr/local/bin/cfssl /usr/local/bin/cfssljson /usr/local/bin/cfssl-certinfo

cat << EOF > ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "876000h"
      }
    }
  }
}
EOF

cat << EOF > ca-csr.json
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

cfssl gencert -initca ca-csr.json | cfssljson -bare ca

# cfssl gencert -initca ca-csr.json | cfssljson -bare ca
2019/12/13 14:35:52 [INFO] generating a new CA key and certificate from CSR
2019/12/13 14:35:52 [INFO] generate received request
2019/12/13 14:35:52 [INFO] received CSR
2019/12/13 14:35:52 [INFO] generating key: rsa-2048
2019/12/13 14:35:52 [INFO] encoded CSR
2019/12/13 14:35:52 [INFO] signed certificate with serial number 248379771349454958117219047414671162179070747780

# 生成文件
-rw-r--r-- 1 root root 1005 12月 13 11:32 ca.csr
-rw------- 1 root root 1675 12月 13 11:32 ca-key.pem
-rw-r--r-- 1 root root 1363 12月 13 11:32 ca.pem
# 配置源文件
-rw-r--r-- 1 root root  293 12月 13 11:31 ca-config.json
-rw-r--r-- 1 root root  210 12月 13 11:31 ca-csr.json

cat << EOF > kubernetes-csr.json
{
  "CN": "kubernetes",
  "hosts": [
    "127.0.0.1",
    "<MASTER_IP>",
    "<MASTER_CLUSTER_IP>", 
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [{
    "C": "<country>",
    "ST": "<state>",
    "L": "<city>",
    "O": "<organization>",
    "OU": "<organization unit>"
  }]
}
EOF

$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes
2019/12/13 14:40:28 [INFO] generate received request
2019/12/13 14:40:28 [INFO] received CSR
2019/12/13 14:40:28 [INFO] generating key: rsa-2048
2019/12/13 14:40:28 [INFO] encoded CSR
2019/12/13 14:40:28 [INFO] signed certificate with serial number 392795299385191732458211386861696542628305189374
2019/12/13 14:40:28 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 生成文件
-rw-r--r-- 1 root root 1269 12月 13 14:40 kubernetes.csr
-rw------- 1 root root 1679 12月 13 14:40 kubernetes-key.pem
-rw-r--r-- 1 root root 1643 12月 13 14:40 kubernetes.pem
# 配置源文件
-rw-r--r-- 1 root root  580 12月 13 14:40 kubernetes-csr.json

cat << EOF > admin-csr.json
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "ShenZhen",
      "L": "ShenZhen",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}
EOF

$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
2019/12/13 14:52:37 [INFO] generate received request
2019/12/13 14:52:37 [INFO] received CSR
2019/12/13 14:52:37 [INFO] generating key: rsa-2048
2019/12/13 14:52:37 [INFO] encoded CSR
2019/12/13 14:52:37 [INFO] signed certificate with serial number 465422983473444224050765004141217688748259757371
2019/12/13 14:52:37 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 生成文件
-rw-r--r-- 1 root root 1013 12月 13 14:52 admin.csr
-rw------- 1 root root 1675 12月 13 14:52 admin-key.pem
-rw-r--r-- 1 root root 1407 12月 13 14:52 admin.pem
# 配置源文件
-rw-r--r-- 1 root root  231 12月 13 14:49 admin-csr.json

cat << EOF > kube-proxy-csr.json
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "BeiJing",
      "L": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

$ cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy

# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes  kube-proxy-csr.json | cfssljson -bare kube-proxy
2019/12/13 19:37:48 [INFO] generate received request
2019/12/13 19:37:48 [INFO] received CSR
2019/12/13 19:37:48 [INFO] generating key: rsa-2048
2019/12/13 19:37:48 [INFO] encoded CSR
2019/12/13 19:37:48 [INFO] signed certificate with serial number 526712749765692443642491255093816136154324531741
2019/12/13 19:37:48 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").

# 生成文件
-rw-r--r-- 1 root root 1009 12月 13 19:37 kube-proxy.csr
-rw------- 1 root root 1675 12月 13 19:37 kube-proxy-key.pem
-rw-r--r-- 1 root root 1407 12月 13 19:37 kube-proxy.pem
# 配置源文件
-rw-r--r-- 1 root root  230 12月 13 19:37 kube-proxy-csr.json

openssl x509  -noout -text -in  kubernetes.pem

# openssl x509  -noout -text -in  kubernetes.pem
Certificate:
    Data:
        Version: 3 (0x2)
        Serial Number:
            44:cd:8c:e6:a4:60:ff:3f:09:af:02:e7:68:5e:f2:0f:e6:a0:39:fe
    Signature Algorithm: sha256WithRSAEncryption
        Issuer: C=CN, ST=ShenZhen, L=ShenZhen, O=k8s, OU=System, CN=kubernetes
        Validity
            Not Before: Dec 13 06:35:00 2019 GMT
            Not After : Nov 19 06:35:00 2119 GMT
        Subject: C=CN, ST=ShenZhen, L=ShenZhen, O=k8s, OU=System, CN=kubernetes
        Subject Public Key Info:
            Public Key Algorithm: rsaEncryption
                Public-Key: (2048 bit)
                Modulus:
                    00:d7:91:4f:90:56:fb:ab:a9:de:c4:98:9e:d7:e6:
                    45:db:5a:14:9a:76:78:6a:4c:db:3c:47:3c:e7:1c:
                    3c:37:4e:8a:cf:9c:a1:8a:4c:51:4c:cd:45:b0:03:
                    87:06:b9:20:2c:3a:51:f9:21:55:1c:90:7c:f8:93:
                    bc:6a:48:05:3d:8b:74:fd:f2:f1:e6:5e:ad:b4:a8:
                    f6:6d:f9:63:9e:e4:b4:cc:68:9e:90:d7:ef:de:ce:
                    c1:1d:1b:68:59:68:5e:5f:7d:5c:f3:49:4f:18:72:
                    be:b5:c8:af:e2:8d:34:9c:d2:68:b7:8c:26:69:cc:
                    a5:f4:ca:69:2d:d7:21:f5:19:2e:b2:b5:97:16:87:
                    9f:9c:fd:01:97:c2:0e:20:b4:88:27:9a:37:9a:af:
                    0a:cf:82:4f:26:24:cb:07:ac:8c:b1:34:20:42:22:
                    00:b2:b0:98:c5:53:01:fb:32:aa:15:1b:7e:39:44:
                    ae:af:6e:c3:65:96:f6:38:7a:87:37:d0:31:63:d8:
                    a4:15:13:f2:56:da:e6:09:45:2b:46:2c:cb:63:db:
                    f7:ba:7f:44:0a:36:39:7c:cc:5b:42:e5:56:c7:7f:
                    dd:64:5c:f2:4a:af:d3:a9:d1:6e:06:27:57:09:4d:
                    db:08:62:87:66:c8:2c:36:00:41:f1:90:f6:5f:68:
                    20:3d
                Exponent: 65537 (0x10001)
        X509v3 extensions:
            X509v3 Key Usage: critical
                Digital Signature, Key Encipherment
            X509v3 Extended Key Usage:
                TLS Web Server Authentication, TLS Web Client Authentication
            X509v3 Basic Constraints: critical
                CA:FALSE
            X509v3 Subject Key Identifier:
                3D:3F:FA:B8:36:D7:FE:B1:59:BE:B1:F5:C1:5D:88:3D:BC:78:9F:87
            X509v3 Authority Key Identifier:
                keyid:40:A2:D4:30:22:12:2E:C2:FB:A2:55:2C:CB:F0:F6:3E:4D:B8:02:03

            X509v3 Subject Alternative Name:
                DNS:kubernetes, DNS:kubernetes.default, DNS:kubernetes.default.svc, DNS:kubernetes.default.svc.cluster, DNS:kubernetes.default.svc.cluster.local, IP Address:127.0.0.1, IP Address:172.20.0.112, IP Address:172.20.0.113, IP Address:172.20.0.114, IP Address:172.20.0.115, IP Address:10.254.0.1
    Signature Algorithm: sha256WithRSAEncryption
         63:50:f6:2a:03:c7:35:dd:e9:10:8d:2f:b3:27:9a:64:f3:e1:
         11:8a:18:1e:fa:6d:85:30:11:b4:59:a3:6c:86:cd:2b:5c:67:
         17:4f:aa:0d:bb:4c:ee:c8:af:e7:3d:61:6d:03:9d:14:6f:00:
         48:56:59:b5:76:13:a9:30:23:e0:b2:d2:12:64:0c:60:0d:76:
         ec:c6:4f:b1:bc:24:01:7a:48:c6:fd:9e:5d:68:da:b9:a1:ad:
         30:7a:ba:90:e2:e3:4e:b4:92:1b:c5:f2:8c:c1:b0:3d:fc:14:
         d2:46:e8:f8:22:8f:d9:4d:85:4f:58:6b:0f:84:78:06:b4:b9:
         92:b9:0d:bd:1d:95:e9:0d:42:d3:fd:dd:2a:59:60:3f:63:35:
         eb:07:25:d2:ea:0d:19:a6:f3:dc:92:8e:ee:73:04:15:5e:97:
         e8:da:51:c3:69:49:96:36:c7:cc:5b:e5:e5:cb:e5:ce:9f:21:
         6f:6b:56:16:bf:85:ad:1c:8c:91:c1:91:0a:90:18:e2:4a:b0:
         32:58:33:ef:55:8e:8f:4a:e3:0f:b8:f7:41:04:65:89:e1:1b:
         d8:41:28:6e:84:c3:1c:8e:a9:a0:8a:42:e4:fe:d7:fe:0e:24:
         dc:74:37:fa:5e:be:20:69:c5:9a:5a:e6:83:1c:0b:9e:e1:43:
         ef:4f:7a:37

mkdir -p /etc/kubernetes/ssl
cp *.pem /etc/kubernetes/ssl

# kubectl get node
Unable to connect to the server: x509: certificate has expired or is not yet valid: current time 2023-08-03T18:06:23+08:00 is after 2023-07-04T06:30:54Z

# 或者出现以下报错
You must be logged in to the server(unauthorized)

kubeadm certs check-expiration

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Dec 30, 2020 23:36 UTC   364d                                    no
apiserver                  Dec 30, 2020 23:36 UTC   364d            ca                      no
apiserver-etcd-client      Dec 30, 2020 23:36 UTC   364d            etcd-ca                 no
apiserver-kubelet-client   Dec 30, 2020 23:36 UTC   364d            ca                      no
controller-manager.conf    Dec 30, 2020 23:36 UTC   364d                                    no
etcd-healthcheck-client    Dec 30, 2020 23:36 UTC   364d            etcd-ca                 no
etcd-peer                  Dec 30, 2020 23:36 UTC   364d            etcd-ca                 no
etcd-server                Dec 30, 2020 23:36 UTC   364d            etcd-ca                 no
front-proxy-client         Dec 30, 2020 23:36 UTC   364d            front-proxy-ca          no
scheduler.conf             Dec 30, 2020 23:36 UTC   364d                                    no

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Dec 28, 2029 23:36 UTC   9y              no
etcd-ca                 Dec 28, 2029 23:36 UTC   9y              no
front-proxy-ca          Dec 28, 2029 23:36 UTC   9y              no

cp -fr /etc/kubernetes /etc/kubernetes.bak
cp -fr ~/.kube ~/.kube.bak

kubeadm certs renew all

# kubeadm certs renew all
[renew] Reading configuration from the cluster...
[renew] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[renew] Error reading configuration from the Cluster. Falling back to default configuration

certificate embedded in the kubeconfig file for the admin to use and for kubeadm itself renewed
certificate for serving the Kubernetes API renewed
certificate the apiserver uses to access etcd renewed
certificate for the API server to connect to kubelet renewed
certificate embedded in the kubeconfig file for the controller manager to use renewed
certificate for liveness probes to healthcheck etcd renewed
certificate for etcd nodes to communicate with each other renewed
certificate for serving etcd renewed
certificate for the front proxy client renewed
certificate embedded in the kubeconfig file for the scheduler manager to use renewed

Done renewing certificates. You must restart the kube-apiserver, kube-controller-manager, kube-scheduler and etcd, so that they can use the new certificates.

crictl ps |grep "etcd"|awk '{print $1}'|xargs crictl stop 

crictl ps |egrep "kube-apiserver|kube-scheduler|kube-controller"|awk '{print $1}'|xargs crictl stop 

cp -fr /etc/kubernetes/admin.conf $HOME/.kube/config

# cat /var/lib/kubelet/config.yaml |grep rotate
rotateCertificates: true

// NewSelfSignedCACert creates a CA certificate
func NewSelfSignedCACert(cfg Config, key crypto.Signer) (*x509.Certificate, error) {
    now := time.Now()
    // returns a uniform random value in [0, max-1), then add 1 to serial to make it a uniform random value in [1, max).
    serial, err := cryptorand.Int(cryptorand.Reader, new(big.Int).SetInt64(math.MaxInt64-1))
    if err != nil {
        return nil, err
    }
    serial = new(big.Int).Add(serial, big.NewInt(1))
    notBefore := now.UTC()
    if !cfg.NotBefore.IsZero() {
        notBefore = cfg.NotBefore.UTC()
    }
    tmpl := x509.Certificate{
        SerialNumber: serial,
        Subject: pkix.Name{
            CommonName:   cfg.CommonName,
            Organization: cfg.Organization,
        },
        DNSNames:              []string{cfg.CommonName},
        NotBefore:             notBefore,
        NotAfter:              now.Add(duration365d * 10).UTC(),   # 默认为10年
        KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign,
        BasicConstraintsValid: true,
        IsCA:                  true,
    }

    certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key)
    if err != nil {
        return nil, err
    }
    return x509.ParseCertificate(certDERBytes)
}